FEXPNetworkServer.h

Go to the documentation of this file.

// © FEXP, FEXPEnterprise Solver, Ing. Vaclav Rek
// Server of the hybrid-parallel FEXP solver.
// Compiler must support C++ ver.14 and later
#ifndef _CFEXPNETWORKSERVER_H_
#define _CFEXPNETWORKSERVER_H_
#include "FEXPCommon.h"
#include "FEXPConcurency.h"
#include "FEXPDataContainer.h"
#include "FEXPGeom.h"
#include "FEXPTopologieKDTree.h"
#include "FEXPMultiGraph.h"
#include "FEXPNetworkInterface.h"
#include "FEXPNetworkWinSocket.h"
#include "FEXPNetworkDataTransferAnalyzer.h"
#include "FEXPResultExport.h"


#define SERVER_STOP_FILE_NAME "stop_fexp_server.stop"

#define SERVER_TRECK_TRNS_DTA "meim_analysis_rs.csv"

// primary template
template<typename TTClientRunner, typename TClientThread, typename Tbarrier, typename Enable = void>
class CFEXPNetServer { };
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
class CFEXPNetServer
  <TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  : public ICFEXPNetNodeConnection, public CFEXPNetWinErrors
{
public:
  virtual ~CFEXPNetServer();
  // create instance of a server -- Singleton pattern
  static Ptr<CFEXPNetServer> GetInstance(std::string server_ip, size_t port,
    Ptr<ICFEXPDataModelContIntf> model, Ptr<ICFEXPResultExport> exporter, Ptr<ICFEXPTransferAnalyzer> trans_analyzer)
  {
    if (SERVER_INSTANCE)
      return SERVER_INSTANCE;
    SERVER_INSTANCE = CFEXPDataManager<CFEXPNetServer>::MakeAllocInstanceSafe(
      new CFEXPNetServer(server_ip, port, model, exporter, trans_analyzer));
    return SERVER_INSTANCE;
  }

  // override pure virtual member functions
  virtual void Run  () override;
  virtual void Close() override;
protected:
  // server single instance
  static Ptr<CFEXPNetServer> SERVER_INSTANCE;
  CFEXPNetServer(std::string server_ip, size_t port,
    Ptr<ICFEXPDataModelContIntf> model, Ptr<ICFEXPResultExport> exporter, Ptr<ICFEXPTransferAnalyzer> trans_analyzer);
  // invoke close action
  void invoke_close();
private:
  Ptr<ICFEXPDataModelContIntf> _model_cont;

  void create_socket    ();
  void start_listenning ();
  void close_socket_conn();
  void cancel_cl_threads();
  // client thread creation
  Ptr<TClientThread> create_client_thread(size_t id, SOCKET client_socket, std::string & ip);
  void               invoke_client_data  ();

  // connection data
  sockaddr_in      _socket_server_adress;
  SOCKET           _socket_server;
  std::atomic_bool _socket_server_run_and_closed;
  // data for each of running client threads
  struct thread_data
  {
    size_t                    _id;
    std::atomic_bool          _state;
    Ptr<TClientThread>        _thread;
    t_ENetThrdMsg             _current_msg_to_send;
    // queue of messages to proccessed
    std::atomic_bool          _message_server_lunched;
    std::queue<t_ENetThrdMsg> _message_queue_server_send;
    std::atomic_bool          _message_client_lunched;
    std::queue<t_ENetThrdMsg> _message_queue_client_send;
    // client node calculation data
    Ptr<t_NetCalcData> _calc_data;
    thread_data(size_t id, Ptr<TClientThread> thread, Ptr<CFEXPSolverConfigSetting> setting) :
      _id(id), _thread(thread), _state(bool(FEXPCOMMON_DEFAULT_VALUE)),
      _current_msg_to_send(t_ENetThrdMsg::eThrdMsgCount),
      _calc_data(CFEXPDataManager<t_NetCalcData>::SafeAllocInstance(setting)),
      _message_server_lunched(true), _message_client_lunched(true) { }
  };

  // data for handling with map of client threads
  volatile bool                      _is_thread_to_remove;
  std::mutex                         _mtx_client_thread_map_update;
  std::atomic_bool                   _flag_server_close;
  std::atomic_bool                   _flag_server_closed;
  std::map<size_t, Ptr<thread_data>> _client_thread_map;
  // synchronization barrier
  enum EBarrierUpdate { eNone, eAddUpdate, eRmvUpdate };
  EBarrierUpdate   _barrier_update;
  std::mutex       _mtx_barrier_update;
  Ptr<Tbarrier>    _barrier           [t_ENetSynStgs::eSynchrCount];
  std::atomic_bool _barrier_synchr_end[t_ENetSynStgs::eSynchrCount];
  std::atomic_bool _barrier_synchr_hit[t_ENetSynStgs::eSynchrCount];
  void update_barrier   (t_ENetSynStgs state);
  void send_msg_and_wait(t_ENetThrdMsg msg, size_t new_thrd_id);
  
  // handling with client threads
  void add_new_client_thread(size_t id, SOCKET client_socket, std::string & ip);
  bool synchronize_process  (size_t id) noexcept(false);
  void set_end_client_thread(size_t id);
  void try_rmv_client_thread();
  // check if all init. clients are connected
  std::atomic_bool _main_proc_initialize_control;
  std::atomic_bool _main_proc_initialize_clients;
  size_t check_main_process_init();
  // sending and receiving messages
  void          send_message(size_t id_from, size_t id_to, t_ENetThrdMsg message);
  t_ENetThrdMsg read_message(size_t id_from, size_t id_to);
  bool          lnch_message(size_t id_from, size_t id_to);
  // get structure of calculation data belonging to client thread
  Ptr<t_NetCalcData> get_calc_data(size_t);
  // stability step resolution process
  t_fexpcommon_ct                        _stabil_dt_step;
  std::mutex                             _mtx_dtstep_update;
  void set_stabil_t_step    (size_t thrd_id, t_fexpcommon_ct step);
  void update_global_t_step (size_t new_thrd_id);
  // analysis of MEIM --> data transfer
  void analyze_data_transfer(size_t new_thrd_id);
  size_t                                 _meim_analysis_counter;
  void write_out_transfer   ();
  // kd-tree of macro elements for assembly of MEIM
  std::mutex                             _mtx_kdtree_update;
  Ptr<ICFEXPTransferAnalyzer>            _transfer_analyzer;
  void add_macro_to_tree(Ptr<CFEXGeomTools::t_BoundBox> bbox, size_t thrd_id, size_t macro_id);
  // map of client node results
  Ptr<ICFEXPResultExport>                _result_exporter;
  bool export_results (size_t new_thrd_id);
  void set_rnt_request();
  // check end of process
  void check_proc_end(size_t new_thrd_id);
};

// member functions
//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
Ptr<CFEXPNetServer<TTClientRunner, TClientThread, Tbarrier>> CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::SERVER_INSTANCE = Ptr<CFEXPNetServer<TTClientRunner, TClientThread, Tbarrier>>();
//--------------------------------------------------------------------------------------

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase, TClientThread>::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::CFEXPNetServer(std::string server_ip, size_t port, Ptr<ICFEXPDataModelContIntf> model, Ptr<ICFEXPResultExport> exporter, Ptr<ICFEXPTransferAnalyzer> trans_analyzer)
  : ICFEXPNetNodeConnection(DEFAULT_SERVER_NODE_ID, server_ip, port),
  _model_cont(model), _result_exporter(exporter), _stabil_dt_step(GET_MAX_FLT_VAL(t_fexpcommon_ct)), _transfer_analyzer(trans_analyzer),
  _socket_server(WININVSCK), _main_proc_initialize_control(false), _main_proc_initialize_clients(false),
  _is_thread_to_remove(false), _socket_server_run_and_closed(false),
  _flag_server_close(false), _flag_server_closed(false),
  _barrier_update(EBarrierUpdate::eNone), _meim_analysis_counter(FEXPCOMMON_DEFAULT_VALUE)
//--------------------------------------------------------------------------------------
{
#define DEFAULT_SERVER_BARRIER_COUNT 1
  size_t idx;
  FEXPCOMMON_FOREACH(size_t(t_ENetSynStgs::eStartPoint ),
                     size_t(t_ENetSynStgs::eSynchrCount) - 1, idx)
  {
    _barrier           [idx] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(DEFAULT_SERVER_BARRIER_COUNT);
    _barrier_synchr_end[idx] = false;
    _barrier_synchr_hit[idx] = false;
  } 
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::~CFEXPNetServer()
//--------------------------------------------------------------------------------------
{
  try
  {
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: Destructing server connection object node ID: " + CFEXPBaseConvers::NumberToString(GetId()) + " ...");
    cancel_cl_threads();
    close_socket_conn();
  }
  catch (const std::exception & ex)
  {
    CFEXPLog::WriteLine();
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "An exception occurred:");
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> " + ex.what());
  }
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::Run()
//--------------------------------------------------------------------------------------
{
#define SERVER_NEW_CLIENT_CONNECT 1
  if (_connection_start.load())
    return;
  // run server thread -- non-blocking accept server --> timeout_ptr != nullptr
  auto result = CFEXPAsyncRunner::RunProcedureTask([this]
    {
      // start listenning on port
      start_listenning();
      // count connected clients -- generate ID
      auto client_id_generator = size_t(FEXPCOMMON_DEFAULT_VALUE);
      auto synchronize_stat_id = size_t(t_ENetSynStgs::eStartPoint); auto new_thrd_id = size_t(FEXPCOMMON_DEFAULT_VALUE);
      auto init_start_clnts_ok = false;
      while (!_flag_server_close.load() && !_socket_server_run_and_closed.load())
      {
        // network calculation synchronization points
        switch (synchronize_stat_id)
        {
        case t_ENetSynStgs::eStartPoint:
          update_barrier(t_ENetSynStgs(synchronize_stat_id));
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eBeforeClientUpdate:
          update_barrier(t_ENetSynStgs(synchronize_stat_id));
          // send current data to clients
          invoke_client_data();
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eClientUpdate:
          {
            fd_set read_set; FD_ZERO(&read_set); FD_SET(_socket_server, &read_set);
            // the maximum time for select to wait, provided in the form of a TIMEVAL structure, 
            timeval timeout;
            timeout.tv_sec   = FEXPCOMMON_DEFAULT_VALUE;
            timeout.tv_usec  = FEXPCOMMON_DEFAULT_VALUE;
            // set the timeout parameter to nullptr for blocking operations
            auto timeout_ptr = &timeout /*nullptr*/;
            // new number of threads for update of thread barrier
            auto new_thr_num = size_t(FEXPCOMMON_DEFAULT_VALUE);
            // determines the status of one or more sockets, waiting if necessary, to perform synchronous I/O.
            if (select(_socket_server, &read_set, nullptr, nullptr, timeout_ptr) != SERVER_NEW_CLIENT_CONNECT)
              try_rmv_client_thread(); // remove ended client thread if it is need
            else
            {
              // new client node connected
              sockaddr_in from; int from_len = sizeof(from);
              new_thrd_id            = ++client_id_generator;
              auto new_client_socket = accept(_socket_server, (struct sockaddr*)&from, &from_len);
              auto new_client_ip_adr = std::string(inet_ntoa(from.sin_addr));
              CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: New client connected:");
              CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> IPv4: " + new_client_ip_adr);
              add_new_client_thread(new_thrd_id, new_client_socket, new_client_ip_adr);
            }
          }
          
          // check if main process can start
          check_main_process_init();
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eAfterClientUpdate:
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eEndPoint:
          update_barrier(t_ENetSynStgs(synchronize_stat_id));
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eCheckCalcEnd:
          update_barrier   (t_ENetSynStgs(synchronize_stat_id));
          // receive info about calculation process
          send_msg_and_wait(t_ENetThrdMsg::eEnd, new_thrd_id);
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eStartNewTimeStep:
          update_barrier(t_ENetSynStgs(synchronize_stat_id));
          // export results --> send last data to export --> end process
          check_proc_end(new_thrd_id);
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eResults:
          update_barrier   (t_ENetSynStgs(synchronize_stat_id));
          // receive calculation results from clients
          send_msg_and_wait(t_ENetThrdMsg::eCalcResults, new_thrd_id);
          // export results
          export_results   (new_thrd_id);
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eStabilityControl:
          update_barrier      (t_ENetSynStgs(synchronize_stat_id));
          // receive stabil time steps from clients
          send_msg_and_wait   (t_ENetThrdMsg::eStep, new_thrd_id);
          // set global time step
          update_global_t_step(new_thrd_id);
          // invoke global time step preparation
          send_msg_and_wait   (t_ENetThrdMsg::eStep, new_thrd_id);
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eDataExchange:
          update_barrier       (t_ENetSynStgs(synchronize_stat_id));
          // receive bounding boxes of macro models
          send_msg_and_wait    (t_ENetThrdMsg::eBoundingBox, new_thrd_id);
          // MEIM analysis
          analyze_data_transfer(new_thrd_id);
          // invoke message again to invoke waiting for data preparation
          send_msg_and_wait    (t_ENetThrdMsg::eBoundingBox, new_thrd_id);
          // --> 1. data from clients to server
          send_msg_and_wait    (t_ENetThrdMsg::eBoundingBox, new_thrd_id);
          // --> 2. data from server to clients
          // synchronize and move to new stage
          if (!synchronize_process(synchronize_stat_id++))
            return;
          break;
        case t_ENetSynStgs::eEndLoop:
          update_barrier(t_ENetSynStgs(synchronize_stat_id));
          // in common situation there are no data to exchange
          if (_main_proc_initialize_control && _main_proc_initialize_clients)
          {
            // reset data analyzer
            if (_transfer_analyzer)
              _transfer_analyzer->Reset();
          }
          // synchronize
          if (!synchronize_process(synchronize_stat_id))
            return;
          // move to new stage (reset value --> new synchronization loop)
          synchronize_stat_id = size_t(t_ENetSynStgs::eStartPoint);
          // reset added last client id
          new_thrd_id         = FEXPCOMMON_DEFAULT_VALUE;
          // reset time step value
          _stabil_dt_step     = GET_MAX_FLT_VAL(t_fexpcommon_ct);
          break;
        default:
          FEXPCOMMON_EXCEPTION("Error: Wrong process synchronization state!!!");
        } 
      }
    }, false, "Server Process", true);
  // invoke close event (stop server)
  invoke_close(); _flag_server_closed.store(true);
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::update_barrier(t_ENetSynStgs state)
//--------------------------------------------------------------------------------------
{
  const size_t distance = size_t(t_ENetSynStgs::eEndPoint) - size_t(t_ENetSynStgs::eClientUpdate);
  switch (state)
  {
  case t_ENetSynStgs::eStartPoint        :
    // the end of permutation update of number of threads in barriers if new thread was added
    if (_barrier_update == EBarrierUpdate::eAddUpdate)
    {
      _barrier[t_ENetSynStgs::eBeforeClientUpdate] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
      _barrier[t_ENetSynStgs::eDataExchange      ] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
    }
    break;
  case t_ENetSynStgs::eBeforeClientUpdate:
    if (_barrier_update == EBarrierUpdate::eAddUpdate)
    {
      _barrier[t_ENetSynStgs::eEndLoop] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
      // reset barrier update control value
      _barrier_update = EBarrierUpdate::eNone;
    }
    break;
  case t_ENetSynStgs::eClientUpdate      :
  case t_ENetSynStgs::eAfterClientUpdate :
  case t_ENetSynStgs::eEndPoint          :
    if (_barrier_update == EBarrierUpdate::eAddUpdate)
      _barrier[state - distance] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
    break;
  case t_ENetSynStgs::eCheckCalcEnd      :
    // if number of thread was changed update number of threads in client control bock
    // --> if new thread was added start permutation update of number of threads in barriers
    if (_barrier_update == EBarrierUpdate::eAddUpdate)
      _barrier[state - distance] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
    break;
  case t_ENetSynStgs::eStartNewTimeStep  :
    // if number of thread was changed update number of threads in client control bock
    // --> if new thread was added start permutation update of number of threads in barriers
    if (_barrier_update == EBarrierUpdate::eAddUpdate)
      _barrier[state - distance] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
    break;
  case t_ENetSynStgs::eResults:
    // continue permutation update of number of threads in barriers if new thread was added
    if (_barrier_update == EBarrierUpdate::eAddUpdate)
      _barrier[state - distance] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
    break;
  case t_ENetSynStgs::eStabilityControl  :
    // continue permutation update of number of threads in barriers if new thread was added
    if (_barrier_update == EBarrierUpdate::eAddUpdate)
      _barrier[state - distance] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
    break;
  case t_ENetSynStgs::eDataExchange      :
    // continue permutation update of number of threads in barriers if new thread was added
    if (_barrier_update == EBarrierUpdate::eAddUpdate)
      _barrier[state - distance] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
    break;
  case t_ENetSynStgs::eEndLoop           :
    // continue permutation update of number of threads in barriers if new thread was added
    if (_barrier_update == EBarrierUpdate::eAddUpdate)
      _barrier[state - distance] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
    break;
  default:
    FEXPCOMMON_EXCEPTION("Error: Unknown type of synchronization phase!!!");
  }
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::send_msg_and_wait(t_ENetThrdMsg msg, size_t new_thrd_id)
//--------------------------------------------------------------------------------------
{
  if (!(_main_proc_initialize_control && _main_proc_initialize_clients))
    return;

  // send data to get current bounding boxes except from last added client
  FEXPCOMMON_FOREACH_ITER(_client_thread_map)
  {
    auto id = (*IT).first;
    if (id == new_thrd_id)
      continue;
    send_message(GetId(), id, msg);
  }
  // wait to read message
  auto msg_read_counter = size_t(FEXPCOMMON_DEFAULT_VALUE);
  while (msg_read_counter != _client_thread_map.size())
  {
    msg_read_counter = size_t(FEXPCOMMON_DEFAULT_VALUE);
    FEXPCOMMON_FOREACH_ITER(_client_thread_map)
    {
      auto id = (*IT).first;
      if (id == new_thrd_id)
        continue;
      if (lnch_message(GetId(), (*IT).first))
        ++msg_read_counter;
    }
    if (new_thrd_id != FEXPCOMMON_DEFAULT_VALUE)
      ++msg_read_counter;
  }
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
Ptr<TClientThread> CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::create_client_thread(size_t id, SOCKET client_socket, std::string & ip)
//--------------------------------------------------------------------------------------
{
  return CFEXPDataManager<TClientThread>::SafeAllocInstance([this](auto client) { client->Run(); },
    CFEXPDataManager<TTClientRunner>::SafeAllocInstance(client_socket, ip, id, GetServerNodeIP(),
      [this](auto id_from, auto id_to           ) { return read_message(id_from, id_to);   },
      [this](auto id_from, auto id_to           ) { return lnch_message(id_from, id_to);   },
      [this](auto id_from, auto id_to, auto mssg) { send_message(id_from, id_to, mssg);    },
      [this](auto id_from                       ) { return get_calc_data(id_from);         },
      [this](auto id_from                       ) { set_end_client_thread(id_from);        },
      [this](auto synstat                       ) { return synchronize_process(synstat);   },
      [this](auto id_from, auto dt              ) { set_stabil_t_step(id_from, dt);        },
      [this](auto id_from, auto bbox , auto mid ) { add_macro_to_tree(bbox, id_from, mid); }));
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::invoke_client_data()
//--------------------------------------------------------------------------------------
{
  Ptr<std::vector<size_t>> schedule; std::queue<size_t> init_models; std::queue<size_t> init_models_schedule;
  FEXPCOMMON_FOREACH_ITER(_client_thread_map)
  {
    auto key = (*IT).first;
    auto dta = (*IT).second;
    auto msg = dta->_current_msg_to_send;
    switch (msg)
    {
    case t_ENetThrdMsg::eInit:
      {
        // server setting
        auto setting = FEXPCOMMON_DYNCAST(ICFEXPModelDataIntf, CFEXPSolverConfigSetting,
          _model_cont->GetModelElement(ICFEXPSetting::ESettingType::eSolver, ESystemElementType::eSetting));
        if (!schedule)
        {
          schedule = setting->GetSimpleModelSchedule();
          // fill queue with models
          FEXPCOMMON_FOREACH_ITER_FNC(setting->GetFileContent(), { init_models.push(IT.first); });
          // fill queue with number of models for each client
          FEXPCOMMON_FOREACH_ITER_FNC(*schedule.get(), { init_models_schedule.push(IT); });
        }
        auto default_model = init_models.empty();
        auto ifile_content = [=](auto id)
          {
            return (default_model ?
              setting->GetFileContentDefault() :
              setting->GetFileContent())[id];
          };
        // set default empty data if no other model is available
        if (default_model)
        {
          FEXPCOMMON_FOREACH_ITER_FNC(setting->GetFileContentDefault(), { init_models.push(IT.first); });
          init_models_schedule.push(init_models.size());
        }
        // add number of models for each client
        size_t idx; size_t mode_count = init_models_schedule.front();
        FEXPCOMMON_FOREACH(FEXPCOMMON_DEFAULT_VALUE, mode_count - 1, idx)
        {
          auto model_id = init_models.front();
          dta->_calc_data->CURRENT_DATA_FOR_SEND.insert(
            MAP_PAIR(CFEXPBaseConvers::NumberToString(model_id), ifile_content(model_id)));
          init_models.pop();
        }
        init_models_schedule.pop();
        // start client initialize completed
        _main_proc_initialize_clients = true;
      }
      break;
    case t_ENetThrdMsg::eRuntimeInit:
      {
        // server setting
        auto setting = FEXPCOMMON_DYNCAST(ICFEXPModelDataIntf, CFEXPSolverConfigSetting,
          _model_cont->GetModelElement(ICFEXPSetting::ESettingType::eSolver, ESystemElementType::eSetting));
        FEXPCOMMON_FOREACH_ITER_FNC(setting->GetFileContentDefault(),
          {
            // send empty data and let client wait for data exchange event
            dta->_calc_data->CURRENT_DATA_FOR_SEND.insert(
              MAP_PAIR(CFEXPBaseConvers::NumberToString(IT.first), IT.second));
          });
      } 
      break;
   default:
      msg = t_ENetThrdMsg::eContinueProc; // continue in process
    }
    // send message to client
    send_message(GetId(), key, msg);
    // reset message
    dta->_current_msg_to_send = t_ENetThrdMsg::eThrdMsgCount;
  };
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::add_new_client_thread(size_t id, SOCKET client_socket, std::string & ip)
//--------------------------------------------------------------------------------------
{
#define MAP_THREAD_IDX       FEXPCOMMON_DEFAULT_INDX
#define MAP_THREAD_VALUE_IDX 1
  if (_client_thread_map.count(id))
    FEXPCOMMON_EXCEPTION("Error: Try add new client thread with existing ID!!!");

  std::unique_lock<std::mutex> lock(_mtx_client_thread_map_update, std::defer_lock);
  lock.lock();
  auto cnt_tm = std::time(nullptr);
  CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: New cluster node ID " + CFEXPBaseConvers::NumberToString(id) + ":");
  CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> Connected at " + std::string(std::ctime(&cnt_tm)));
  // create new client thread data, start thread and detach
  auto server_thread_data = CFEXPDataManager<thread_data>::SafeAllocInstance(id, create_client_thread(id, client_socket, ip),
    FEXPCOMMON_DYNCAST(ICFEXPModelDataIntf, CFEXPSolverConfigSetting,
      _model_cont->GetModelElement(ICFEXPSetting::ESettingType::eSolver, ESystemElementType::eSetting)));
  // add thread data to map
  _client_thread_map.insert(MAP_PAIR(server_thread_data->_id, server_thread_data));
  // update number of threads in eStartPoint barrier only
  // --> permutation process
  _barrier[t_ENetSynStgs::eStartPoint] =
    CFEXPDataManager<Tbarrier>::SafeAllocInstance(_client_thread_map.size() + 1);
  // start thread and detach
  server_thread_data->_thread->StartThread();
  server_thread_data->_thread->Detach();
  // set active state
  server_thread_data->_state = true;
  // if calculation is running and new client is connected --> data are required
  server_thread_data->_current_msg_to_send = _main_proc_initialize_control ? t_ENetThrdMsg::eRuntimeInit : t_ENetThrdMsg::eContinueProc;
  // need update barriers
  _barrier_update = EBarrierUpdate::eAddUpdate;
  // set request for runtime data printing
  set_rnt_request();
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
bool CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::synchronize_process(size_t id) noexcept(false)
//--------------------------------------------------------------------------------------
{
  if(id >= t_ENetSynStgs::eSynchrCount)
    FEXPCOMMON_EXCEPTION("Error: Wrong process synchronization state!!!");
  if (_barrier_synchr_end[id].load())
    return false;
  _barrier_synchr_hit[id].store(true );
  _barrier           [id]->SynchronizeThreads();
  _barrier_synchr_hit[id].store(false);
  return true;
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::set_end_client_thread(size_t id)
//--------------------------------------------------------------------------------------
{
  std::unique_lock<std::mutex> lock(_mtx_client_thread_map_update, std::defer_lock);
  lock.lock();
  if (_client_thread_map.count(id))
  {
    auto cnt_tm = std::time(nullptr);
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: Cluster node ID " + CFEXPBaseConvers::NumberToString(id) + ":");
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> Ended at " + std::string(std::ctime(&cnt_tm)));
    _client_thread_map[id]->_state = !(_is_thread_to_remove = true);
  }
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::try_rmv_client_thread()
//--------------------------------------------------------------------------------------
{
  std::unique_lock<std::mutex> lock(_mtx_client_thread_map_update, std::defer_lock);
  lock.lock();
  if (_is_thread_to_remove)
  {
    std::vector<size_t> to_remove;
    FEXPCOMMON_FOREACH_ITER_FNC(_client_thread_map,
      {
        if (!IT.second->_state)
          to_remove.push_back(IT.first);
      });
    size_t idx; auto endidx = to_remove.size() - 1;
    FEXPCOMMON_FOREACH(FEXPCOMMON_DEFAULT_INDX, endidx, idx)
    {
      auto cnt_tm = std::time(nullptr);
      auto trd_id = to_remove[idx];
      // remove thread data from map
      _client_thread_map.erase(_client_thread_map.find(trd_id));
      CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: Cluster node ID " + CFEXPBaseConvers::NumberToString(trd_id) + ":");
      CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> Removed at " + std::string(std::ctime(&cnt_tm)));
    }
    _is_thread_to_remove = false;
    // check if need to update barriers
    _barrier_update = !to_remove.empty() ? EBarrierUpdate::eRmvUpdate : EBarrierUpdate::eNone;
    // update number of threads in barriers if it is need
    if (_barrier_update == EBarrierUpdate::eRmvUpdate)
    {
      auto tread_count = _client_thread_map.size() + 1;
      _barrier[t_ENetSynStgs::eStartPoint       ] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(tread_count);
      _barrier[t_ENetSynStgs::eAfterClientUpdate] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(tread_count);
      _barrier[t_ENetSynStgs::eEndPoint         ] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(tread_count);
      _barrier[t_ENetSynStgs::eCheckCalcEnd     ] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(tread_count);
      _barrier[t_ENetSynStgs::eStartNewTimeStep ] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(tread_count);
      _barrier[t_ENetSynStgs::eResults          ] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(tread_count);
      _barrier[t_ENetSynStgs::eStabilityControl ] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(tread_count);
      _barrier[t_ENetSynStgs::eDataExchange     ] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(tread_count);
      _barrier[t_ENetSynStgs::eEndLoop          ] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(tread_count);
    } 
  }
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
size_t CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::check_main_process_init()
//--------------------------------------------------------------------------------------
{
  auto result = size_t(FEXPCOMMON_DEFAULT_VALUE);
  if (_main_proc_initialize_control)
    return result;

  std::unique_lock<std::mutex> lock(_mtx_client_thread_map_update, std::defer_lock);
  lock.lock();
  size_t counter = FEXPCOMMON_DEFAULT_VALUE;
  FEXPCOMMON_FOREACH_ITER_FNC(_client_thread_map,
    {
      if (IT.second->_state)
        ++counter;
    });
  // get server setting
  auto setting = FEXPCOMMON_DYNCAST(ICFEXPModelDataIntf, CFEXPSolverConfigSetting,
    _model_cont->GetModelElement(ICFEXPSetting::ESettingType::eSolver, ESystemElementType::eSetting));
  // check number of clients to start calculation
  result = setting->GetInitClientNumber() - counter;
  _main_proc_initialize_control.store(!_main_proc_initialize_control ? result == FEXPCOMMON_DEFAULT_VALUE : _main_proc_initialize_control);
  lock.unlock();
  // print out start main process
  if (_main_proc_initialize_control)
  {
    // add init message to process
    FEXPCOMMON_FOREACH_ITER_FNC(_client_thread_map, { IT.second->_current_msg_to_send = t_ENetThrdMsg::eInit; });
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: Initial number of cluster nodes has been connected:");
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> Calculation will start immediately!");
  }
  return result;
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::send_message(size_t id_from, size_t id_to, t_ENetThrdMsg message)
//--------------------------------------------------------------------------------------
{
  if (message == t_ENetThrdMsg::eThrdMsgCount)
    return;

  std::unique_lock<std::mutex> lock(_mtx_client_thread_map_update, std::defer_lock);
  lock.lock();
  if (GetId() == id_from)
  {
    // messages from server to client
    auto thread = _client_thread_map[id_to];
    if (thread->_message_client_lunched)
      thread->_message_client_lunched.store(false);
    thread->_message_queue_server_send.push(message);
  }
  else
  {
    // messages from client to server
    auto thread = _client_thread_map[id_from];
    if (thread->_message_server_lunched)
      thread->_message_server_lunched.store(false);
    thread->_message_queue_client_send.push(message);
  }
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
bool CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::lnch_message(size_t id_from, size_t id_to)
//--------------------------------------------------------------------------------------
{
  return GetId() == id_from ?
    _client_thread_map[id_to  ]->_message_client_lunched.load() :
    _client_thread_map[id_from]->_message_server_lunched.load();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
t_ENetThrdMsg CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::read_message(size_t id_from, size_t id_to)
//--------------------------------------------------------------------------------------
{
  t_ENetThrdMsg message;
  std::unique_lock<std::mutex> lock(_mtx_client_thread_map_update, std::defer_lock);
  lock.lock();
  if (GetId() == id_from)
  {
    auto thread = _client_thread_map[id_to];
    if (!thread->_message_client_lunched.load() && !thread->_message_queue_server_send.empty())
    {
      message = thread->_message_queue_server_send.front();
      thread->_message_queue_server_send.pop();
      if (thread->_message_queue_server_send.empty())
        thread->_message_client_lunched.store(true);
    }
    else
      message = t_ENetThrdMsg::eThrdMsgCount;
  }
  else
  {
    auto thread = _client_thread_map[id_from];
    if (!thread->_message_server_lunched.load() && !thread->_message_queue_client_send.empty())
    {
      message = thread->_message_queue_client_send.front();
      thread->_message_queue_client_send.pop();
      if (thread->_message_queue_client_send.empty())
        thread->_message_server_lunched.store(true);
    }
    else
      message = t_ENetThrdMsg::eThrdMsgCount;
  }
  lock.unlock();
  return message;
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
Ptr<t_NetCalcData> CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::get_calc_data(size_t id)
//--------------------------------------------------------------------------------------
{
  Ptr<t_NetCalcData> data;
  std::unique_lock<std::mutex> lock(_mtx_client_thread_map_update, std::defer_lock);
  lock.lock();
  if (_client_thread_map.count(id))
    data = _client_thread_map[id]->_calc_data;
  lock.unlock();
  return data;
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
bool CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::export_results(size_t new_thrd_id)
//--------------------------------------------------------------------------------------
{
  auto result_exported = false;
  if (!(_main_proc_initialize_control && _main_proc_initialize_clients))
    return result_exported;

  std::map<size_t, Ptr<CFEXPCalculationModelNodeResult>> thread_result_map;
  FEXPCOMMON_FOREACH_ITER(_client_thread_map)
  {
    auto thrd_id = (*IT).first;
    if (thrd_id == new_thrd_id)
      continue;
    auto results = (*IT).second->_calc_data->CURRENT_DATA_RECEIVED;
    if (results.empty())
      continue;

    // print out runtime calculation data
    auto resulto_rows = results["rso"];
    if (resulto_rows && !resulto_rows->empty())
    {
      auto output = resulto_rows->at(FEXPCOMMON_DEFAULT_INDX);
      if (output != FEXPCOMMON_EMPTY_STRING)
        CFEXPLog::WriteLine(output);
    }

    auto result_rows = results["rs"];
    if (!result_rows || result_rows->empty())
      continue;
    // read serialized results
    auto reader = CFEXPDataManager<CFEXPFileReader<CFEXResultDataContainer>>
      ::SafeAllocInstance(FEXPCOMMON_EMPTY_STRING, result_rows, CFEXPResultDataModelAssemblyFactory::RES_FILE_BLOCK_CLS_MAP);
    reader->Read();
    // build result instance
    CFEXPDataManager<CFEXPModelBuilder<ICFEXPDataContIntf, ICFEXPDataModelContIntf>>
      ::SafeAllocInstance(_model_cont, FEXPCOMMON_EMPTY_STRING, FEXPCOMMON_EMPTY_STRING, FEXPCOMMON_EMPTY_STRING)->BuildModel(
        CFEXPBaseConvers::NumberToString(thrd_id), reader->GetFileContent(),
      FEXPCOMMON_DYNCAST(CFEXPFEInpContBase, ICFEXPDataContIntf, reader->GetInputContainer()));
    if (_model_cont->ItemCount(ESystemElementType::eResult) > 1)
      FEXPCOMMON_EXCEPTION("Error: Data from more time levels arrived!!!");
    // add to result map
    _model_cont->IterateModElems([&thread_result_map, &thrd_id](auto item)
      { thread_result_map.insert(MAP_PAIR(thrd_id, FEXPCOMMON_STACAST(ICFEXPModelDataIntf, CFEXPCalculationModelNodeResult, item))); return true; },
        ESystemElementType::eResult);

    // clear lunched result data
    results.clear();
    // clear results from current client and prepare for next one
    _model_cont->RemoveAll(ESystemElementType::eResult);
  }
  if (thread_result_map.empty())
    return result_exported;

  Ptr<CFEXPCalculationModelNodeResult> results;
  FEXPCOMMON_FOREACH_ITER(thread_result_map)
  {
    if (!results)
    {
      results = (*IT).second;
      continue;
    }
    // node results
    FEXPCOMMON_FOREACH_ITER_FNC((*IT).second->GetNodeResults (), { results->GetNodeResults ().push_back(IT); });
    // fe connectivity
    FEXPCOMMON_FOREACH_ITER_FNC((*IT).second->GetConnectivity(), { results->GetConnectivity().push_back(IT); });
  }

  // final export of results
  if (_result_exporter)
    _result_exporter->Write(results);
  return !result_exported;
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::set_rnt_request()
//--------------------------------------------------------------------------------------
{
  // server setting
  auto setting = FEXPCOMMON_DYNCAST(ICFEXPModelDataIntf, CFEXPSolverConfigSetting,
    _model_cont->GetModelElement(ICFEXPSetting::ESettingType::eSolver, ESystemElementType::eSetting));

  auto dkey = "rso";
  FEXPCOMMON_FOREACH_ITER(_client_thread_map)
  {
    if (!IT->second->_calc_data->CURRENT_DATA_FOR_POUT.empty())
      continue;
    IT->second->_calc_data->CURRENT_DATA_FOR_POUT
      .insert(MAP_PAIR(dkey, CFEXPDataManager<std::vector<std::string>>::SafeAllocInstance()));
    IT->second->_calc_data->CURRENT_DATA_FOR_POUT[dkey]->push_back(CFEXPBaseConvers::NumberToString(setting->NodeIdToShow ()));
    IT->second->_calc_data->CURRENT_DATA_FOR_POUT[dkey]->push_back(CFEXPBaseConvers::NumberToString(setting->NodeDofToShow()));
  }
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::set_stabil_t_step(size_t thrd_id, t_fexpcommon_ct step)
//--------------------------------------------------------------------------------------
{
  std::unique_lock<std::mutex> lock(_mtx_dtstep_update, std::defer_lock);
  lock.lock();
  _stabil_dt_step = step < _stabil_dt_step ? step : _stabil_dt_step;
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::update_global_t_step(size_t new_thrd_id)
//--------------------------------------------------------------------------------------
{
  if (!(_main_proc_initialize_control && _main_proc_initialize_clients))
    return;
  auto dkey = "dt";
  FEXPCOMMON_FOREACH_ITER(_client_thread_map)
  {
    if (IT->first == new_thrd_id)
      continue;
    IT->second->_calc_data->CURRENT_DATA_FOR_SEND
      .insert(MAP_PAIR(dkey, CFEXPDataManager<std::vector<std::string>>::SafeAllocInstance()));
    IT->second->_calc_data->CURRENT_DATA_FOR_SEND[dkey]->push_back(CFEXPBaseConvers::NumberToString(_stabil_dt_step));
  }
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::add_macro_to_tree(Ptr<CFEXGeomTools::t_BoundBox> bbox, size_t thrd_id, size_t macro_id)
//--------------------------------------------------------------------------------------
{
  if (!_transfer_analyzer)
    return;
  std::unique_lock<std::mutex> lock(_mtx_kdtree_update, std::defer_lock);
  lock.lock();
  // add macro bounding box into the MEIM analyzer
  _transfer_analyzer->AddMacro(bbox, macro_id, thrd_id);
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::analyze_data_transfer(size_t new_thrd_id)
//--------------------------------------------------------------------------------------
{
  if (!(_main_proc_initialize_control && _main_proc_initialize_clients))
    return;
  ++_meim_analysis_counter;
  // clear data from previous session
  FEXPCOMMON_FOREACH_ITER_FNC(_client_thread_map,
    {
      IT.second->_calc_data->MODEL_DATA_GET.clear();
      IT.second->_calc_data->MODEL_DATA_SET.clear();
    });
  // analyze data transfer
  if (!_transfer_analyzer->Analyze())
    return;
  // save data from analysis for data transfer
  auto data_from  = _transfer_analyzer->GetTransferComputerModelFrom ();
  auto data_to    = _transfer_analyzer->GetTransferComputerModelTo   ();
  auto data_assoc = _transfer_analyzer->GetMacroToComputerAssociation();
  if      (data_from.empty() && data_to.empty())
    return;
  else if (data_from.empty() || data_to.empty())
    FEXPCOMMON_EXCEPTION("Error: Wrong data transfer analysis!!!");
  CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: Transfer of data required.");
  // store data for data transfer
  FEXPCOMMON_FOREACH_ITER(_client_thread_map)
  {
    auto tid = IT->first;
    auto dta = IT->second->_calc_data;
    if (data_from.count(tid))
      FEXPCOMMON_FOREACH_ITER_FNC(data_from[tid], { dta->MODEL_DATA_GET.push_back(IT); });
    if (data_to.count(tid))
      FEXPCOMMON_FOREACH_ITER_FNC(data_to  [tid], { dta->MODEL_DATA_SET.push_back(std::make_tuple(IT, data_assoc[IT])); });
  };
  // try to write current transfer data to file
  write_out_transfer();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::write_out_transfer()
//--------------------------------------------------------------------------------------
{
  std::ofstream file;
  file.open(SERVER_TRECK_TRNS_DTA, std::ios::out | std::ios::app);
  if (file.fail())
    return;
  file << "MEIM analysis count (transfer required): " << _meim_analysis_counter << std::endl;
  // transfer "from"
  FEXPCOMMON_FOREACH_ITER(_client_thread_map)
  {
    auto tid = IT->first;
    auto dta = IT->second->_calc_data;
    if (dta->MODEL_DATA_GET.empty())
      continue;
    file << "Data from: c" << tid << std::endl;
    FEXPCOMMON_FOREACH_ITER_FNC(dta->MODEL_DATA_GET,
      { auto value = IT; file << "m" << value << FEXPCOMMON_DELIMITER; });
    file << std::endl;
  }
  // transfer "to"
  FEXPCOMMON_FOREACH_ITER(_client_thread_map)
  {
    auto tid = IT->first;
    auto dta = IT->second->_calc_data;
    if (dta->MODEL_DATA_SET.empty())
      continue;
    file << "Data to: c" << tid << std::endl;
    FEXPCOMMON_FOREACH_ITER_FNC(dta->MODEL_DATA_SET,
      { file << "m" << std::get<0>(IT) << FEXPCOMMON_DELIMITER << "c" << std::get<1>(IT) << std::endl; });
  }
  file.close();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::check_proc_end(size_t new_thrd_id)
//--------------------------------------------------------------------------------------
{
  if (export_results(new_thrd_id))
  {
    send_msg_and_wait(t_ENetThrdMsg::eEnd, new_thrd_id);
    size_t idx;
    FEXPCOMMON_FOREACH(size_t(t_ENetSynStgs::eStartPoint), size_t(t_ENetSynStgs::eSynchrCount) - 1, idx)
      _barrier_synchr_end[idx].store(true);
  }
  else
    send_msg_and_wait(t_ENetThrdMsg::eContinueProc, new_thrd_id);
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::Close()
//--------------------------------------------------------------------------------------
{
#define ERROR_SOCKET_SERVER_CLOSE_CONNECT "Error: Server connection has already been closed: \n"
#define ERROR_SOCKET_SERVER_CLOSE_RUNCLSE "Error: Try close server which was run and closed!!!"
  if(_socket_server_run_and_closed)
    FEXPCOMMON_EXCEPTION(ERROR_SOCKET_SERVER_CLOSE_RUNCLSE);
  if (!_connection_start.load())
  {
    _socket_server_run_and_closed.store(true);
    return;
  }

  // set close flag of closing server
  _flag_server_close.store(true); std::vector<Ptr<CFEXPCppThread<void(size_t), size_t>>> temp_thread_vector;
  auto result = CFEXPAsyncRunner::RunProcedureTask([this, &temp_thread_vector]
    {
      // set end of process (do not use synchronization --> start ending processes)
      size_t idx;
      FEXPCOMMON_FOREACH(size_t(t_ENetSynStgs::eStartPoint ),
                         size_t(t_ENetSynStgs::eSynchrCount) - 1, idx)
        _barrier_synchr_end[idx].store(true);
      // wait until server main loop end
      while (!_flag_server_closed.load())
      {
        // wake up all waiting threads
        FEXPCOMMON_FOREACH(size_t(t_ENetSynStgs::eStartPoint ),
                           size_t(t_ENetSynStgs::eSynchrCount) - 1, idx)
        {
          // check if any thread wait on respective barrier
          if (!_barrier_synchr_hit[idx].load())
            continue;
          // start temporary thread to increase thread count on barrier and thus invoke the end event (end of waiting within the barrier)
          auto tmpi = idx;
          auto temp = CFEXPDataManager<CFEXPCppThread<void(size_t), size_t>>
            ::SafeAllocInstance([this](auto id) { _barrier[id]->SynchronizeThreads(); }, tmpi);
          temp->StartThread(); temp->Detach();
          // thread must "start" and "end" before temporary thread variable is destroyed due to no reference
          temp_thread_vector.push_back(temp);
        }
      }
    }, false, "Waiting for closing of server main thread", true);
  // release all temporary threads
  temp_thread_vector.clear();
  // cancel client threads
  cancel_cl_threads();
  // close and clean up
  close_socket_conn();
  _socket_server_run_and_closed.store(true);
  SERVER_INSTANCE = Ptr<CFEXPNetServer<TTClientRunner, TClientThread, Tbarrier>>();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::invoke_close()
//--------------------------------------------------------------------------------------
{
  // create stop file to invoke server close
  (std::ofstream(SERVER_STOP_FILE_NAME)).close();
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::create_socket()
//--------------------------------------------------------------------------------------
{
#define ERROR_SERVER_SOCKET_CREATION "Error: Cannot create socket (server): \n"
  // information about the Windows Sockets implementation
  WSADATA socket_info;
  // initiates use of the Winsock DLL by a process
  auto result = WSAStartup(WINSOCKET, &socket_info);
  if (CONNECTION_ERRORS_1.count(result))
    FEXPCOMMON_EXCEPTION(ERROR_SERVER_SOCKET_CREATION + CONNECTION_ERRORS_1[result]);

  // fill server info for connection
  memset(&_socket_server_adress, FEXPCOMMON_DEFAULT_VALUE, sizeof(_socket_server_adress));
  _socket_server_adress.sin_family      = WINNETWRK;
  _socket_server_adress.sin_addr.s_addr = WINANYADR;
  _socket_server_adress.sin_port        = htons((u_short)GetCommPort());
  // create server socket
  _socket_server = socket(WINNETWRK, WINSCTYPE, WINPR_TCP/*WINPR_DMY*/);
  if (CONNECTION_ERRORS_2.count(result))
    FEXPCOMMON_EXCEPTION(ERROR_SERVER_SOCKET_CREATION + CONNECTION_ERRORS_2[result]);

  // associating a local address with a socket.
  result = bind(_socket_server, (sockaddr*)&_socket_server_adress, sizeof(_socket_server_adress));
  if (CONNECTION_ERRORS_4.count(result))
    FEXPCOMMON_EXCEPTION(ERROR_SERVER_SOCKET_CREATION + CONNECTION_ERRORS_4[result]);

  /*// non-blocking recv on socket
  struct timeval tv;
  tv.tv_sec = 10;
  if (setsockopt(_socket_server, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof tv))
  ;*/
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
  <TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::start_listenning()
//--------------------------------------------------------------------------------------
{
#define ERROR_SOCKET_SERVER_CONNECT "Error: Server cannot start: \n"
  if (_socket_server_run_and_closed)
  {
    auto error = ERROR_SOCKET_SERVER_CONNECT +
                 std::string("--> Node ID") + "\"" + CFEXPBaseConvers::NumberToString<size_t>(GetId()) + "\":\n" +
                 std::string("--> Server closed.");
    FEXPCOMMON_EXCEPTION(error);
  }

  // create socket and connect
  create_socket();
  // places a socket in a state in which it is listening for an incoming connection
  auto result = listen(_socket_server, 10);
  if (CONNECTION_ERRORS_5.count(result))
  {
    // throw error connection exception
    auto error = ERROR_SOCKET_SERVER_CONNECT +
                 std::string("--> Node ID ") + "\"" + CFEXPBaseConvers::NumberToString<size_t>(GetId()) + "\":\n";
    error     += CONNECTION_ERRORS_4[result];
    FEXPCOMMON_EXCEPTION(error);
  }
  // set server connection started
  _connection_start.store(true);
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::close_socket_conn()
//--------------------------------------------------------------------------------------
{
  if (_socket_server == WININVSCK)
    return;
  closesocket(_socket_server);
  WSACleanup();
  _socket_server = WININVSCK;
}

//--------------------------------------------------------------------------------------
template<typename TTClientRunner, typename TClientThread, typename Tbarrier>
void CFEXPNetServer
<TTClientRunner, TClientThread, Tbarrier,
  typename std::enable_if<std::is_base_of<ICFEXPThreadBase         , TClientThread>::value &&
                          std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier     >::value>::type>
  ::cancel_cl_threads()
//--------------------------------------------------------------------------------------
{
  if (_client_thread_map.empty())
    return;
  // canceling procedure
  auto result = CFEXPAsyncRunner::RunProcedureTask([this]
    {
      // cancel all clients threads
      FEXPCOMMON_FOREACH_ITER_FNC(_client_thread_map,
        {
          if (IT.second->_state)
            _client_thread_map[IT.first]->_thread->Cancel();
        });
      // wait for end of all client threads
      while (true)
      {
        size_t cancel_thread_counter = FEXPCOMMON_DEFAULT_VALUE;
        std::unique_lock<std::mutex> lock(_mtx_client_thread_map_update, std::defer_lock);
        lock.lock();
        FEXPCOMMON_FOREACH_ITER_FNC(_client_thread_map,
          {
            if (!IT.second->_state)
              ++cancel_thread_counter;
          });
        lock.unlock();
        // check if all threads have already been closed
        if (cancel_thread_counter == _client_thread_map.size())
          break;
      }
    }, false, "Try cancel server client threads", true);
  // clear thread maps
  _client_thread_map.clear();
}

#endif // !_CFEXPNETWORKSERVER_H_